{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 用LightGBM完成任务，并用交叉验证对模型的超参数（learning_rate、n_estimators、num_leaves、max_depth、min_data_in_leaf、colsample_bytree、subsample）进行调优。（70分） "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 导入相应工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Monthly_Income</th>\n",
       "      <th>DOB</th>\n",
       "      <th>Loan_Amount_Applied</th>\n",
       "      <th>Loan_Tenure_Applied</th>\n",
       "      <th>Existing_EMI</th>\n",
       "      <th>Employer_Name</th>\n",
       "      <th>Var5</th>\n",
       "      <th>Loan_Amount_Submitted</th>\n",
       "      <th>...</th>\n",
       "      <th>Var4_5</th>\n",
       "      <th>Var4_6</th>\n",
       "      <th>Var4_7</th>\n",
       "      <th>var1_B</th>\n",
       "      <th>var1_C</th>\n",
       "      <th>var1_D</th>\n",
       "      <th>var1_E</th>\n",
       "      <th>var1_F</th>\n",
       "      <th>var1_H</th>\n",
       "      <th>var1_I</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.555654</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.251908</td>\n",
       "      <td>1.097388</td>\n",
       "      <td>-0.693912</td>\n",
       "      <td>-0.743281</td>\n",
       "      <td>-3.866817e-16</td>\n",
       "      <td>-0.231402</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.304683</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.523794</td>\n",
       "      <td>-1.386852</td>\n",
       "      <td>-0.693912</td>\n",
       "      <td>3.414388</td>\n",
       "      <td>1.207754e+00</td>\n",
       "      <td>-0.488816</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.374578</td>\n",
       "      <td>2</td>\n",
       "      <td>0.563751</td>\n",
       "      <td>0.269308</td>\n",
       "      <td>-0.693912</td>\n",
       "      <td>-0.743281</td>\n",
       "      <td>-3.866817e-16</td>\n",
       "      <td>0.308643</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.304683</td>\n",
       "      <td>2</td>\n",
       "      <td>1.651297</td>\n",
       "      <td>1.097388</td>\n",
       "      <td>-0.693912</td>\n",
       "      <td>-0.229697</td>\n",
       "      <td>5.547067e-01</td>\n",
       "      <td>1.807866</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.918651</td>\n",
       "      <td>2</td>\n",
       "      <td>0.291865</td>\n",
       "      <td>-1.386852</td>\n",
       "      <td>1.717598</td>\n",
       "      <td>-0.064359</td>\n",
       "      <td>2.078484e+00</td>\n",
       "      <td>0.468135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 69 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Gender  Monthly_Income  DOB  Loan_Amount_Applied  \\\n",
       "0           0       0       -0.555654    2            -0.251908   \n",
       "1           1       1        0.304683    2            -0.523794   \n",
       "2           2       1       -0.374578    2             0.563751   \n",
       "3           3       1        0.304683    2             1.651297   \n",
       "4           4       1        1.918651    2             0.291865   \n",
       "\n",
       "   Loan_Tenure_Applied  Existing_EMI  Employer_Name          Var5  \\\n",
       "0             1.097388     -0.693912      -0.743281 -3.866817e-16   \n",
       "1            -1.386852     -0.693912       3.414388  1.207754e+00   \n",
       "2             0.269308     -0.693912      -0.743281 -3.866817e-16   \n",
       "3             1.097388     -0.693912      -0.229697  5.547067e-01   \n",
       "4            -1.386852      1.717598      -0.064359  2.078484e+00   \n",
       "\n",
       "   Loan_Amount_Submitted   ...    Var4_5  Var4_6  Var4_7  var1_B  var1_C  \\\n",
       "0              -0.231402   ...         0       0       0       0       1   \n",
       "1              -0.488816   ...         0       0       0       0       0   \n",
       "2               0.308643   ...         0       0       0       0       1   \n",
       "3               1.807866   ...         0       0       0       0       1   \n",
       "4               0.468135   ...         0       0       0       0       1   \n",
       "\n",
       "   var1_D  var1_E  var1_F  var1_H  var1_I  \n",
       "0       0       0       0       0       0  \n",
       "1       0       0       0       1       0  \n",
       "2       0       0       0       0       0  \n",
       "3       0       0       0       0       0  \n",
       "4       0       0       0       0       0  \n",
       "\n",
       "[5 rows x 69 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "train= pd.read_csv(dpath +\"bank_data.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Disbursed'] \n",
    "X_train = train.drop([\"LoggedIn\", \"Disbursed\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ROUNDS = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 先调 n_estimators参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_n_estimators(params , X_train , y_train , early_stopping_rounds=10):\n",
    "    lgbm_params = params.copy()\n",
    "#     lgbm_params['num_class'] = 2\n",
    "    lgbmtrain = lgbm.Dataset(X_train , y_train )\n",
    "     \n",
    "    #num_boost_round为弱分类器数目，下面的代码参数里因为已经设置了early_stopping_rounds\n",
    "    #即性能未提升的次数超过过早停止设置的数值，则停止训练\n",
    "    cv_result = lgbm.cv(lgbm_params , lgbmtrain , num_boost_round=MAX_ROUNDS , nfold=3,  metrics='binary_logloss' , early_stopping_rounds=early_stopping_rounds,seed=3 )\n",
    "    cv_result_frame=pd.DataFrame(cv_result)\n",
    "    print(cv_result_frame)\n",
    "    print('best n_estimators:' , len(cv_result['binary_logloss-mean']))\n",
    "    print('best cv score:' , cv_result['binary_logloss-mean'][-1])\n",
    "     \n",
    "    return len(cv_result['binary_logloss-mean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    binary_logloss-mean  binary_logloss-stdv\n",
      "0              0.078077             0.000235\n",
      "1              0.076577             0.000239\n",
      "2              0.075769             0.000314\n",
      "3              0.074793             0.000354\n",
      "4              0.074132             0.000316\n",
      "5              0.073470             0.000354\n",
      "6              0.072923             0.000418\n",
      "7              0.072432             0.000440\n",
      "8              0.071985             0.000472\n",
      "9              0.071608             0.000477\n",
      "10             0.071301             0.000498\n",
      "11             0.071029             0.000480\n",
      "12             0.070930             0.000478\n",
      "13             0.070702             0.000472\n",
      "14             0.070625             0.000409\n",
      "15             0.070441             0.000383\n",
      "16             0.070217             0.000378\n",
      "17             0.070069             0.000354\n",
      "18             0.069916             0.000346\n",
      "19             0.069791             0.000352\n",
      "20             0.069673             0.000360\n",
      "21             0.069565             0.000344\n",
      "22             0.069497             0.000330\n",
      "23             0.069362             0.000336\n",
      "24             0.069231             0.000334\n",
      "25             0.069231             0.000298\n",
      "26             0.069195             0.000272\n",
      "27             0.069111             0.000309\n",
      "28             0.069043             0.000353\n",
      "29             0.069012             0.000333\n",
      "..                  ...                  ...\n",
      "37             0.068771             0.000353\n",
      "38             0.068707             0.000305\n",
      "39             0.068734             0.000331\n",
      "40             0.068700             0.000321\n",
      "41             0.068755             0.000313\n",
      "42             0.068748             0.000320\n",
      "43             0.068725             0.000324\n",
      "44             0.068732             0.000343\n",
      "45             0.068748             0.000373\n",
      "46             0.068665             0.000312\n",
      "47             0.068654             0.000332\n",
      "48             0.068641             0.000370\n",
      "49             0.068603             0.000310\n",
      "50             0.068609             0.000312\n",
      "51             0.068587             0.000314\n",
      "52             0.068600             0.000296\n",
      "53             0.068601             0.000313\n",
      "54             0.068584             0.000282\n",
      "55             0.068601             0.000289\n",
      "56             0.068597             0.000310\n",
      "57             0.068605             0.000314\n",
      "58             0.068608             0.000320\n",
      "59             0.068645             0.000316\n",
      "60             0.068605             0.000266\n",
      "61             0.068618             0.000238\n",
      "62             0.068625             0.000256\n",
      "63             0.068583             0.000212\n",
      "64             0.068578             0.000214\n",
      "65             0.068563             0.000208\n",
      "66             0.068543             0.000197\n",
      "\n",
      "[67 rows x 2 columns]\n",
      "best n_estimators: 67\n",
      "best cv score: 0.06854328172792161\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'n_jobs': -1,\n",
    "          'learning_rate': 0.1,\n",
    "          'num_leaves': 60,\n",
    "          'max_depth': 6,\n",
    "          'max_bin': 127,\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "\n",
    "n_estimators_1 = get_n_estimators(params , X_train , y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### num_leaves & max_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 tasks      | elapsed:   16.1s\n",
      "[Parallel(n_jobs=-1)]: Done  43 out of  48 | elapsed:   29.2s remaining:    3.3s\n",
      "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed:   29.8s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_bin=127, max_depth=-1, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=67,\n",
       "        n_jobs=-1, num_leaves=31, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=0.7,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'num_leaves': range(50, 90, 10), 'max_depth': range(5, 9)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,#数据样本不均衡\n",
    "          'n_jobs': -1,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,#上面训练得到的参数\n",
    "#         'max_depth': 7,\n",
    "          'max_bin': 127,\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(50,90,10) #50,60,70,80\n",
    "max_depth=range(5,9,1)\n",
    "tuned_parameters = dict( num_leaves = num_leaves_s,max_depth=max_depth)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=-1, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2694208268819263\n",
      "{'max_depth': 8, 'num_leaves': 80}\n"
     ]
    }
   ],
   "source": [
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### max_depth= 8, num_leaves=80"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### min_child_samples 叶子节点的最小样本数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   8 out of  12 | elapsed:    7.4s remaining:    3.7s\n",
      "[Parallel(n_jobs=4)]: Done  12 out of  12 | elapsed:    9.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_bin=127, max_depth=8, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=67,\n",
       "        n_jobs=-1, num_leaves=80, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=0.7,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'min_child_samples': range(10, 50, 10)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'n_jobs': -1,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,#上面训练得到的参数\n",
    "          'max_depth': 8,\n",
    "          'num_leaves':80,\n",
    "          'max_bin': 127,\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "min_child_samples_s = range(10,50,10) \n",
    "tuned_parameters = dict( min_child_samples = min_child_samples_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=-1,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.26201828175686975\n",
      "{'min_child_samples': 10}\n"
     ]
    }
   ],
   "source": [
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### min_child_samples=10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 行采样参数 sub_samples/bagging_fraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of  15 | elapsed:    3.5s remaining:    9.7s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of  15 | elapsed:    6.7s remaining:    5.8s\n",
      "[Parallel(n_jobs=-1)]: Done  12 out of  15 | elapsed:    8.5s remaining:    2.1s\n",
      "[Parallel(n_jobs=-1)]: Done  15 out of  15 | elapsed:    8.8s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_bin=127, max_depth=8, min_child_samples=10,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=67,\n",
       "        n_jobs=-1, num_leaves=80, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'subsample': [0.5, 0.6, 0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'n_jobs': -1,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,#上面训练得到的参数\n",
    "          'max_depth': 8,\n",
    "          'num_leaves':80,\n",
    "          'min_child_samples':10,\n",
    "          'max_bin': 127,\n",
    "#           'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "subsample_s = [i/10.0 for i in range(5,10)]\n",
    "tuned_parameters = dict( subsample = subsample_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=-1,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.26201828175686975\n",
      "{'subsample': 0.7}\n"
     ]
    }
   ],
   "source": [
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### subsample=0.7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 列采样参数 sub_feature/feature_fraction/colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of  15 | elapsed:    2.7s remaining:    7.6s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of  15 | elapsed:    2.9s remaining:    2.5s\n",
      "[Parallel(n_jobs=-1)]: Done  12 out of  15 | elapsed:    5.2s remaining:    1.2s\n",
      "[Parallel(n_jobs=-1)]: Done  15 out of  15 | elapsed:    5.3s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_bin=127, max_depth=8, min_child_samples=10,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=67,\n",
       "        n_jobs=-1, num_leaves=80, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=0.7,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'colsample_bytree': [0.5, 0.6, 0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'n_jobs': -1,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,#上面训练得到的参数\n",
    "          'max_depth': 8,\n",
    "          'num_leaves':80,\n",
    "          'min_child_samples':10,\n",
    "          'max_bin': 127,\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "#           'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "colsample_bytree_s = [i/10.0 for i in range(5,10)]\n",
    "tuned_parameters = dict( colsample_bytree = colsample_bytree_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=-1,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.25862954763374146\n",
      "{'colsample_bytree': 0.9}\n"
     ]
    }
   ],
   "source": [
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### colsample_bytree=0.9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 用所有训练数据，采用最佳参数重新训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.9, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_bin=127, max_depth=8, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=67,\n",
       "        n_jobs=-1, num_leaves=80, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=0.7,\n",
       "        subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'n_jobs': -1,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,\n",
    "          'max_depth': 8,\n",
    "          'num_leaves':80,\n",
    "          'min_child_samples':20,\n",
    "          'max_bin': 127,\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.9,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)# 这里没有交叉验证了，所以样本数目增多。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 对最终模型给出特征重要性（10分）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Monthly_Income</td>\n",
       "      <td>789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Employer_Name</td>\n",
       "      <td>522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Existing_EMI</td>\n",
       "      <td>403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Var5</td>\n",
       "      <td>385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Loan_Amount_Submitted</td>\n",
       "      <td>354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Loan_Amount_Applied</td>\n",
       "      <td>313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Loan_Tenure_Applied</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Loan_Tenure_Submitted</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Filled_Form</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Salary_Account_other</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Salary_Account_HDFC Bank</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>City_O</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Var1_C</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>City_D</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Var4_3</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>City_P</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>City_H</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>City_B</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Var4_2</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Salary_Account_State Bank of India</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Source_S133</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Var1_D</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Salary_Account_ICICI Bank</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DOB</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gender</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Var1_B</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>City_M</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>City_C</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Source_S122</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Var1_E</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Var4_7</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Var4_1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Var2_C</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Salary_Account_Citibank</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Device_Type</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>City_K</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Var2_G</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Salary_Account_Kotak Bank</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Var1_H</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Var2_E</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Salary_Account_IDBI Bank</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Var4_5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>City_J</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Source_S127</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Salary_Account_Punjab National Bank</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Source_S137</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>City_Co</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Var2_D</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Var1_F</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>var1_H</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Var4_6</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>var1_C</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Var2_F</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>City_G</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>var1_B</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>var1_D</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>var1_F</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>var1_E</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>var1_I</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>67 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                columns  importance\n",
       "2                        Monthly_Income         789\n",
       "0                            Unnamed: 0         663\n",
       "7                         Employer_Name         522\n",
       "6                          Existing_EMI         403\n",
       "8                                  Var5         385\n",
       "9                 Loan_Amount_Submitted         354\n",
       "4                   Loan_Amount_Applied         313\n",
       "5                   Loan_Tenure_Applied         137\n",
       "10                Loan_Tenure_Submitted          85\n",
       "11                          Filled_Form          72\n",
       "31                 Salary_Account_other          71\n",
       "25             Salary_Account_HDFC Bank          64\n",
       "22                               City_O          63\n",
       "34                               Var1_C          62\n",
       "16                               City_D          61\n",
       "55                               Var4_3          57\n",
       "23                               City_P          55\n",
       "18                               City_H          52\n",
       "13                               City_B          52\n",
       "54                               Var4_2          50\n",
       "30   Salary_Account_State Bank of India          49\n",
       "48                          Source_S133          47\n",
       "35                               Var1_D          47\n",
       "26            Salary_Account_ICICI Bank          46\n",
       "3                                   DOB          46\n",
       "1                                Gender          45\n",
       "33                               Var1_B          44\n",
       "21                               City_M          38\n",
       "14                               City_C          37\n",
       "46                          Source_S122          32\n",
       "..                                  ...         ...\n",
       "36                               Var1_E          25\n",
       "59                               Var4_7          25\n",
       "53                               Var4_1          23\n",
       "41                               Var2_C          22\n",
       "24              Salary_Account_Citibank          20\n",
       "12                          Device_Type          19\n",
       "20                               City_K          18\n",
       "45                               Var2_G          16\n",
       "28            Salary_Account_Kotak Bank          15\n",
       "38                               Var1_H          14\n",
       "43                               Var2_E          14\n",
       "27             Salary_Account_IDBI Bank          14\n",
       "57                               Var4_5          12\n",
       "19                               City_J          12\n",
       "47                          Source_S127          11\n",
       "29  Salary_Account_Punjab National Bank           9\n",
       "50                          Source_S137           8\n",
       "15                              City_Co           8\n",
       "42                               Var2_D           8\n",
       "37                               Var1_F           6\n",
       "65                               var1_H           6\n",
       "58                               Var4_6           5\n",
       "61                               var1_C           5\n",
       "44                               Var2_F           3\n",
       "17                               City_G           3\n",
       "60                               var1_B           3\n",
       "62                               var1_D           3\n",
       "64                               var1_F           2\n",
       "63                               var1_E           1\n",
       "66                               var1_I           1\n",
       "\n",
       "[67 rows x 2 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"columns\":list(feat_names), \"importance\":list(lg.feature_importances_.T)})\n",
    "df = df.sort_values(by=['importance'],ascending=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
